
"""

"""
# $Id: extract_datetime.py 32 2011-08-04 02:09:03Z postshift@gmail.com $

import re
import sys
import time
import dateutil.parser as dparser
from datetime import datetime
import traceback
import wikinbitDb
from BeautifulSoup import BeautifulSoup

from optparse import OptionParser

parser = OptionParser(usage="")
parser.add_option("--num_names", dest="desired_num_names", type=int)
(options, args) = parser.parse_args()

javascript_re = re.compile(r'<script[^<>]*?>.*?</script>', re.M)
style_re = re.compile(r'<style[^<>]*?>.*?</style>', re.M)

DEFAULT = datetime(2020, 1, 2)

# we will use this DB cursor many times below --- commiting each time
cur = wikinbitDb.conn.cursor()
num_names = 0
while num_names < options.desired_num_names:
    try:
        cur.execute("""
            select refUrl, content from refs where 
              length(content) > 0 and 
              not exists
                (select * from refs_times where refs_times.refUrl = refs.refUrl) 
              limit 1""")
        (refUrl, content) = cur.fetchall()[0]
        assert content
    except Exception, exc:
        sys.exit(traceback.format_exc(exc))

    #print len(content) 

    # Brutally remove script and style tags instead of ignoring them
    # in the parsetree, because beautifulsoup's findAll call below is
    # not behaving as I expect.
    content = " ".join(content.splitlines())
    content = javascript_re.sub("", content)
    content = style_re.sub("", content)
    #print "After deleting javascript, we have %d bytes of content" % len(content)
    try:
        soup = BeautifulSoup(content)
    except Exception, exc:
        print traceback.format_exc(exc)
        print "Looping through."
        soup = None

    foundTime = None
    if soup:
        for navString in soup.findAll(
            #re.compile("!(script|style)", re.I),   # not sure why this fails
            text = re.compile("\S+")):

            try:
                # this does a lame job of parsing unstructured text.  How
                # should we improve it?
                foundTime = dparser.parse(navString, fuzzy=True, default=DEFAULT)
                
                #print "%s --> %s" % (navString, foundTime)
                if foundTime.year < 1970 or foundTime.year > 2011:
                    foundTime = None
                    continue

                if foundTime == DEFAULT:
                    foundTime = None
                    continue
                # Take the first one we get!
                break
            except Exception, exc:
                # when this fails, which it does frequently, just give up
                # and log a tracback.
                #print traceback.format_exc(exc)
                continue

    try:
        navString = navString.encode("utf8")
    except Exception, exc:
        foundTime = None

    if not foundTime:
        foundTime = 0
        navString = ""
    else:
        # put it into seconds since the epoch, so we have an int.
        foundTime = int(time.strftime("%s", foundTime.timetuple()))

    cur.execute("""
            insert into refs_times(refUrl, foundTime, timeString)
            values (%s, %s, %s)""",
                    (refUrl, foundTime, navString))

    print "Foundtime:  %s --> %d" % (navString, foundTime)
    sys.stdout.flush()

    wikinbitDb.conn.commit()

    num_names += 1

wikinbitDb.conn.close()
